package shining365.programmableweb.data.collect;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;

import javax.annotation.Resource;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.transform.stream.StreamSource;

import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.Serializer;
import net.sf.saxon.s9api.XdmNode;
import net.sf.saxon.s9api.XsltCompiler;
import net.sf.saxon.s9api.XsltExecutable;
import net.sf.saxon.s9api.XsltTransformer;

import org.apache.commons.configuration.Configuration;
import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import shining365.programmableweb.data.common.Utils;
import shining365.programmableweb.data.model.Mashup;
import shining365.programmableweb.data.model.Mashups;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations={"/applicationContext.xml"})
public class CollectData {

	@Test
	@Rollback(value=false)
	public void collectData() throws IOException, DocumentException {
		Foxtidy foxtidy = new Foxtidy();
		
		boolean done = false;
		final int startPage = config.containsKey("mashup.startPage") ? config
				.getInt("mashup.startPage") : 1;
		final int endPage = config.containsKey("mashup.endPage") ? config
				.getInt("mashup.endPage") : Integer.MAX_VALUE;
		for (int i = startPage; i <= endPage && !done; i++) {
			String url = "http://www.programmableweb.com/mashups/directory/" 
				+ i + "?view=desc&sort=date&pagesize=50";
			Document document = foxtidy.getPageDocument(url);
			
			byte[] xmlBean = transform(document.asXML().getBytes(), "xml/Mashup.xslt");
			Mashups mashups = (Mashups)xml2Object(xmlBean);
			
			if (mashups.getMashup().size() != 50) {
				logger.warn("nodes number: " + mashups.getMashup().size() + " url:" + url);
				done = true;
			}
			for (Mashup mashup : mashups.getMashup()) {
				if (mashup.getApi().size() == 0) {
					logger.error("api size=0, mashup: " + mashup.getName());
				}
				if (dao.findMashup(mashup.getName()) != null) {
					logger.warn("duplicate mashup: " + mashup.getName());
				}
				dao.merge(mashup);
			}
		}
		
	}

	private static byte[] transform(byte[] srcXml, String xsltFilePath) {
		//输入输出流
		ByteArrayInputStream is = new ByteArrayInputStream(srcXml);
		ByteArrayOutputStream os = new ByteArrayOutputStream();
	
		try {
			//编译
			Processor proc = new Processor(false);
			XsltCompiler comp = proc.newXsltCompiler();
			XsltExecutable exp = comp.compile(
					new StreamSource(new File(xsltFilePath)));
	
			//准备输入输出
			XdmNode source = proc.newDocumentBuilder().build(new StreamSource(is));
			Serializer serializer = new Serializer();
			serializer.setOutputProperty(Serializer.Property.METHOD, "xml");
			serializer.setOutputProperty(Serializer.Property.INDENT, "yes");
			serializer.setOutputStream(os);
	
			//转换
			XsltTransformer trans = exp.load();
			trans.setInitialContextNode(source);
			trans.setDestination(serializer);
			trans.transform();
		} catch (SaxonApiException e) {
			logger.error("saxon xslt error", e);
		}
	    
	    return os.toByteArray();
	}

	private static Object xml2Object(byte[] xml) {
		try {
			JAXBContext jc = JAXBContext.newInstance(
				"shining365.programmableweb.data.model");
			Unmarshaller unmarshaller = jc.createUnmarshaller();
			ByteArrayInputStream xmlStream = new ByteArrayInputStream(xml);
			return unmarshaller.unmarshal(xmlStream);
		} catch (JAXBException e) {
			logger.error("JAXB error", e);
			return null;
		}
	}

	@Resource
	private Dao dao;
	private static final Configuration config = Utils.getConfiguration();
	private static final Logger	logger = Logger.getLogger(CollectData.class);
}
